package com.fansl.allround.upms.helper.region;

import cn.hutool.core.util.ReUtil;
import cn.hutool.core.util.StrUtil;
import com.fansl.allround.common.core.constant.enums.RegionLevelEnum;
import com.fansl.allround.upms.api.entity.SysRegion;
import com.fansl.allround.upms.helper.RegionHelper;
import com.fansl.allround.upms.service.SysRegionService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.RedisTemplate;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;

import static com.fansl.allround.upms.helper.RegionHelper.HELP_URL;

/**
 * @author fansl
 * @Description: TODO
 * @date 2019/8/23 16:32
 */
@Slf4j
public class ProvinceProcessor implements PageProcessor {
    /**
     * 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
     */
    private final Site site = Site.me()
            .setCharset("gbk").setTimeOut(100 * 1000)
            .setRetryTimes(3).setSleepTime(3000)
            .setCycleRetryTimes(3);
    private final SysRegionService sysRegionService;
    private final RedisTemplate redisTemplate;
    private final boolean recursionFlag;

    public ProvinceProcessor(SysRegionService sysRegionService,
                             RedisTemplate redisTemplate,
                             boolean recursionFlag) {
        this.sysRegionService = sysRegionService;
        this.redisTemplate = redisTemplate;
        this.recursionFlag = recursionFlag;
    }

    @Override
    /**
     * process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
     */
    public void process(Page page) {
        Html html = page.getHtml();
        // 部分二：定义如何抽取页面信息，并保存下来
        List<Selectable> provinceHtmlList = html.xpath("//tr[@class='provincetr']/td/a").nodes();
        //省份信息
        List<SysRegion> provinceList = new ArrayList<>();
        List<String> provinceUrlList = new ArrayList<>();
        provinceHtmlList.stream().forEach(selectable -> {
            String linkUrl = selectable.$("a", "href").get();
            String code = StrUtil.fillAfter(StrUtil.removeAll(linkUrl, ".html"), "0".charAt(0), 12);
            String pageUrl = page.getUrl().get();
            linkUrl = ReUtil.replaceAll(pageUrl, "\\w+.html", linkUrl);
            String name = selectable.$("a", "text").get();
            log.info("省份行政编码：{}", code);
            log.info("省份名称：{}", name);
            log.info("省份下的链接：{}", linkUrl);
            SysRegion province = new SysRegion();
            province.setParentCode("CN")
                    .setName(name)
                    .setCode(code)
                    .setLevel(RegionLevelEnum.PROVINCE.getLevel())
                    .setNextLevelUrl(linkUrl);
            RegionHelper.setRegionAlias(province);
            provinceList.add(province);
            if (StrUtil.isNotBlank(linkUrl)) {
                provinceUrlList.add(linkUrl);
            }
        });
        page.putField(RegionLevelEnum.PROVINCE.name(), provinceList);
        //保存到redis中
        page.putField(HELP_URL, page.getUrl());
        if (recursionFlag) {
            //异步执行
            CompletableFuture.runAsync(() -> {
                //获取下一级数据
                if (provinceUrlList.isEmpty()) {
                    return;
                }
                RegionHelper.executeSpider(RegionLevelEnum.CITY,
                        provinceUrlList, recursionFlag, sysRegionService, redisTemplate);
            });
        }
    }

    @Override
    public Site getSite() {
        return site;
    }
}
